#!/usr/bin/env python

import nltk
from nltk.corpus import brown
from nltk import FreqDist
from nltk import bigrams
from nltk import ConditionalFreqDist
from collections import defaultdict

brown_news_tagged = brown.tagged_words(categories='news',tagset='universal')

# text = 'And now for something completely different'
# tokens = nltk.word_tokenize(text)
# print nltk.pos_tag(tokens)


# text = nltk.Text(w.lower() for w in brown.words())
# print text.similar('woman')


# tagged_fd = FreqDist(tag for (w,tag) in brown_news_tagged)
# print tagged_fd.most_common()
# tagged_fd.plot()


# word_noun = [w for (w,tag) in brown_news_tagged if tag=='NOUN']
# print word_noun[:10]


# word_tag_pairs = bigrams(brown_news_tagged)
# non_preceders =[a[1] for(a,b) in word_tag_pairs if b[1] =='NOUN']
# fdist = FreqDist(non_preceders)
# print fdist.most_common()
# fdist.plot()


# cfd = ConditionalFreqDist(
# 	(w.lower(), tag)
# 	for (w,tag) in brown_news_tagged
# 	)
# for w in sorted(cfd.conditions()):
# 	if len(cfd[w]) > 3:
# 		tags = [tag for (tag,_) in cfd[w].most_common()]
# 		print w, ' '.join(tags)



# counts = defaultdict(int)
# for (word,tag) in brown_news_tagged:
# 	counts[tag] += 1
# print counts['NOUN']
# print counts
# from operator import itemgetter
# print sorted(counts.items(), key=itemgetter(1), reverse=True)


# words_all = brown.words(categories='news')
# same_chars = defaultdict(list)
# for w in words_all:
# 	key = w[:3]
# 	same_chars[key].append(w)
# print same_chars['exp']


# -------------- Anagrams
# words_all = brown.words(categories='news')
# same_chars = defaultdict(list)
# for w in words_all:
# 	key = ''.join(sorted(w))
# 	if w not in same_chars[key]:
# 		same_chars[key].append(w)
# print same_chars['aet']